{
  "annotations": {
    "list": [
      {
        "builtIn": 1,
        "datasource": "$ds_prometheus",
        "enable": true,
        "hide": true,
        "iconColor": "rgba(0, 211, 255, 1)",
        "name": "Annotations & Alerts",
        "target": {
          "limit": 100,
          "matchAny": false,
          "tags": [],
          "type": "dashboard"
        },
        "type": "dashboard"
      }
    ]
  },
  "description": "Metrics useful for benchmarking and loadtesting Prometheus itself.",
  "editable": false,
  "fiscalYearStartMonth": 0,
  "gnetId": 12054,
  "graphTooltip": 1,
  "id": 33,
  "iteration": 1667582638340,
  "links": [],
  "liveNow": true,
  "panels": [
    {
      "collapsed": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 0
      },
      "id": 49,
      "panels": [],
      "title": "Basics",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "orange",
            "mode": "fixed"
          },
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 5,
        "x": 0,
        "y": 1
      },
      "id": 40,
      "links": [],
      "options": {
        "colorMode": "value",
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {},
        "textMode": "name"
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "editorMode": "code",
          "exemplar": false,
          "expr": "max by (pod, version) (prometheus_build_info{job=\"prometheus\", pod=\"$pod\"})",
          "format": "time_series",
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "{{version}}",
          "range": false,
          "refId": "A"
        }
      ],
      "title": "Prometheus Version",
      "transparent": true,
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "yellow",
            "mode": "fixed"
          },
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          }
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 8,
        "x": 5,
        "y": 1
      },
      "id": 109,
      "links": [],
      "options": {
        "colorMode": "value",
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "text": {},
        "textMode": "name"
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "editorMode": "code",
          "exemplar": false,
          "expr": "max by (pod, revision) (prometheus_build_info{job=\"prometheus\", pod=\"$pod\"})",
          "format": "time_series",
          "instant": true,
          "intervalFactor": 1,
          "legendFormat": "{{revision}}",
          "range": false,
          "refId": "A"
        }
      ],
      "title": "Prometheus Revision",
      "transparent": true,
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "blue",
            "mode": "fixed"
          },
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "dtdurations"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 5,
        "x": 13,
        "y": 1
      },
      "id": 72,
      "links": [],
      "options": {
        "colorMode": "value",
        "graphMode": "none",
        "justifyMode": "auto",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "max by (pod) (time() - process_start_time_seconds{job=\"prometheus\",pod=\"$pod\"})",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Age",
          "refId": "A"
        }
      ],
      "title": "Uptime",
      "transparent": true,
      "type": "stat"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "fixedColor": "purple",
            "mode": "fixed"
          },
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "dtdurations"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 3,
        "w": 6,
        "x": 18,
        "y": 1
      },
      "id": 107,
      "links": [],
      "options": {
        "colorMode": "value",
        "graphMode": "none",
        "justifyMode": "center",
        "orientation": "auto",
        "reduceOptions": {
          "calcs": [
            "lastNotNull"
          ],
          "fields": "",
          "values": false
        },
        "textMode": "auto"
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "max by (pod) (time() - prometheus_config_last_reload_success_timestamp_seconds{job=\"prometheus\",pod=\"$pod\"})",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Age",
          "refId": "A"
        }
      ],
      "title": "Last Successful Config Reload",
      "transparent": true,
      "type": "stat"
    },
    {
      "collapsed": false,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 4
      },
      "id": 55,
      "panels": [],
      "title": "Resource Usage",
      "type": "row"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Allocated bytes"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#7EB26D",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Allocated bytes - 1m max"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#BF1B00",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Allocated bytes - 1m min"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#BF1B00",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Allocated bytes - 5m max"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#BF1B00",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Allocated bytes - 5m min"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#BF1B00",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Chunks"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#1F78C1",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Chunks to persist"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#508642",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Max chunks"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#052B51",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Max to persist"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#3F6833",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "RSS"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#447EBC",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byRegexp",
              "options": "/-/"
            },
            "properties": [
              {
                "id": "custom.fillOpacity",
                "value": 0
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 5
      },
      "id": 6,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "max by (pod) (process_resident_memory_bytes{job=\"prometheus\",pod=\"$pod\"})",
          "intervalFactor": 2,
          "legendFormat": "RSS",
          "metric": "process_resident_memory_bytes",
          "refId": "B",
          "step": 10
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "max by (pod) (go_memstats_next_gc_bytes{job=\"prometheus\",pod=\"$pod\"})",
          "intervalFactor": 2,
          "legendFormat": "Next GC",
          "metric": "go_memstats_next_gc_bytes",
          "refId": "C",
          "step": 10
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "max by (pod) (go_memstats_alloc_bytes{job=\"prometheus\",pod=\"$pod\"})",
          "intervalFactor": 2,
          "legendFormat": "Allocated",
          "metric": "go_memstats_alloc_bytes",
          "refId": "A",
          "step": 10
        }
      ],
      "title": "Memory",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Allocated bytes"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#F9BA8F",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Chunks"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#1F78C1",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Chunks to persist"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#508642",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Max chunks"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#052B51",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Max to persist"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#3F6833",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "RSS"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#890F02",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 5
      },
      "id": 7,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "max by (pod) (rate(go_memstats_alloc_bytes_total{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
          "intervalFactor": 2,
          "legendFormat": "Allocated Bytes/s",
          "metric": "go_memstats_alloc_bytes",
          "refId": "A",
          "step": 10
        }
      ],
      "title": "Allocations",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "none"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 5
      },
      "id": 9,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "hidden",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "single",
          "sort": "none"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "editorMode": "code",
          "expr": "max by (pod) (irate(process_cpu_seconds_total{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
          "intervalFactor": 2,
          "legendFormat": "__auto",
          "metric": "prometheus_local_storage_ingested_samples_total",
          "range": true,
          "refId": "A",
          "step": 10
        }
      ],
      "title": "CPU",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 0,
        "y": 12
      },
      "id": 70,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "max by (pod) (prometheus_tsdb_symbol_table_size_bytes{job=\"prometheus\",pod=\"$pod\"})",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "RAM Used",
          "metric": "prometheus_local_storage_ingested_samples_total",
          "refId": "A",
          "step": 10
        }
      ],
      "title": "Symbol Tables Size",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 10,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "min": 0,
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "bytes"
        },
        "overrides": []
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 8,
        "y": 12
      },
      "id": 71,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "max by (pod) (prometheus_tsdb_storage_blocks_bytes_total{job=\"prometheus\",pod=\"$pod\"} or prometheus_tsdb_storage_blocks_bytes{job=\"prometheus\",pod=\"$pod\"})",
          "format": "time_series",
          "intervalFactor": 2,
          "legendFormat": "Disk Used",
          "metric": "prometheus_local_storage_ingested_samples_total",
          "refId": "A",
          "step": 10
        }
      ],
      "title": "Block Size",
      "type": "timeseries"
    },
    {
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "fieldConfig": {
        "defaults": {
          "color": {
            "mode": "palette-classic"
          },
          "custom": {
            "axisLabel": "",
            "axisPlacement": "auto",
            "barAlignment": 0,
            "drawStyle": "line",
            "fillOpacity": 0,
            "gradientMode": "none",
            "hideFrom": {
              "legend": false,
              "tooltip": false,
              "viz": false
            },
            "lineInterpolation": "linear",
            "lineWidth": 1,
            "pointSize": 5,
            "scaleDistribution": {
              "type": "linear"
            },
            "showPoints": "never",
            "spanNulls": false,
            "stacking": {
              "group": "A",
              "mode": "none"
            },
            "thresholdsStyle": {
              "mode": "off"
            }
          },
          "links": [],
          "mappings": [],
          "thresholds": {
            "mode": "absolute",
            "steps": [
              {
                "color": "green",
                "value": null
              },
              {
                "color": "red",
                "value": 80
              }
            ]
          },
          "unit": "short"
        },
        "overrides": [
          {
            "matcher": {
              "id": "byName",
              "options": "Max"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#e24d42",
                  "mode": "fixed"
                }
              }
            ]
          },
          {
            "matcher": {
              "id": "byName",
              "options": "Open"
            },
            "properties": [
              {
                "id": "color",
                "value": {
                  "fixedColor": "#508642",
                  "mode": "fixed"
                }
              }
            ]
          }
        ]
      },
      "gridPos": {
        "h": 7,
        "w": 8,
        "x": 16,
        "y": 12
      },
      "id": 41,
      "links": [],
      "options": {
        "legend": {
          "calcs": [],
          "displayMode": "list",
          "placement": "bottom"
        },
        "tooltip": {
          "mode": "multi",
          "sort": "none"
        }
      },
      "pluginVersion": "8.5.13",
      "targets": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "max by (pod) (process_max_fds{job=\"prometheus\",pod=\"$pod\"})",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Max",
          "refId": "A"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "expr": "max by (pod) (process_open_fds{job=\"prometheus\",pod=\"$pod\"})",
          "format": "time_series",
          "intervalFactor": 1,
          "legendFormat": "Open",
          "refId": "B"
        }
      ],
      "title": "File Descriptors",
      "type": "timeseries"
    },
    {
      "collapsed": true,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 19
      },
      "id": 46,
      "panels": [
        {
          "datasource": "$ds_prometheus",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "short"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Time series"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#70dbed",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 0,
            "y": 20
          },
          "id": 3,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "hidden",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod) (prometheus_tsdb_head_series{job=\"prometheus\",pod=\"$pod\"})",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "Time series",
              "metric": "prometheus_local_storage_memory_series",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Head Time series",
          "type": "timeseries"
        },
        {
          "datasource": "$ds_prometheus",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "short"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 8,
            "y": 20
          },
          "id": 26,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "hidden",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod) (prometheus_tsdb_head_active_appenders{job=\"prometheus\",pod=\"$pod\"})",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "Head Appenders",
              "metric": "prometheus_local_storage_memory_series",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Head Active Appenders",
          "type": "timeseries"
        },
        {
          "datasource": "$ds_prometheus",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "short"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "samples/s"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#e5a8e2",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 16,
            "y": 20
          },
          "id": 1,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "hidden",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod) (rate(prometheus_tsdb_head_samples_appended_total{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "samples/s",
              "metric": "prometheus_local_storage_ingested_samples_total",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Samples Appended/s",
          "type": "timeseries"
        },
        {
          "datasource": "$ds_prometheus",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "short"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "To persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#9AC48A",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byRegexp",
                  "options": "/Max.*/"
                },
                "properties": [
                  {
                    "id": "custom.fillOpacity",
                    "value": 0
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 0,
            "y": 27
          },
          "id": 2,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "hidden",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod) (prometheus_tsdb_head_chunks{job=\"prometheus\",pod=\"$pod\"})",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "Chunks",
              "metric": "prometheus_local_storage_memory_chunks",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Head Chunks",
          "type": "timeseries"
        },
        {
          "datasource": "$ds_prometheus",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "ops"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 8,
            "y": 27
          },
          "id": 4,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod) (rate(prometheus_tsdb_head_chunks_created_total{job=\"prometheus\",instance=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "Created",
              "metric": "prometheus_local_storage_chunk_ops_total",
              "refId": "A",
              "step": 10
            },
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod) (rate(prometheus_tsdb_head_chunks_removed_total{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "Removed",
              "metric": "prometheus_local_storage_chunk_ops_total",
              "refId": "B",
              "step": 10
            }
          ],
          "title": "Head Chunks/s",
          "type": "timeseries"
        },
        {
          "datasource": "$ds_prometheus",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "short"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Removed"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#e5ac0e",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 16,
            "y": 27
          },
          "id": 25,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod) (prometheus_tsdb_isolation_high_watermark{job=\"prometheus\",pod=\"$pod\"} - prometheus_tsdb_isolation_low_watermark{job=\"prometheus\",pod=\"$pod\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "Difference",
              "metric": "prometheus_local_storage_chunk_ops_total",
              "refId": "B",
              "step": 10
            }
          ],
          "title": "Isolation Watermarks",
          "type": "timeseries"
        }
      ],
      "title": "Ingestion",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 20
      },
      "id": 91,
      "panels": [
        {
          "datasource": "$ds_prometheus",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "short"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 10,
            "w": 24,
            "x": 0,
            "y": 35
          },
          "id": 42,
          "links": [],
          "options": {
            "legend": {
              "calcs": [
                "lastNotNull",
                "max",
                "min"
              ],
              "displayMode": "table",
              "placement": "right"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "editorMode": "code",
              "expr": "max by (name, config) (prometheus_sd_discovered_targets{job=\"prometheus\", pod=\"$pod\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{name}}-{{config}}",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Discovered Targets Number",
          "type": "timeseries"
        },
        {
          "datasource": "$ds_prometheus",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "short"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 7,
            "w": 13,
            "x": 0,
            "y": 45
          },
          "id": 96,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "hidden",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "editorMode": "code",
              "expr": "max by (pod) (rate(prometheus_sd_updates_total{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "__auto",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Sent Updates/s",
          "type": "timeseries"
        },
        {
          "datasource": "$ds_prometheus",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "short"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 7,
            "w": 11,
            "x": 13,
            "y": 45
          },
          "id": 97,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "hidden",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "editorMode": "code",
              "expr": "max by (pod) (rate(prometheus_sd_received_updates_total{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "__auto",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Received Updates/s",
          "type": "timeseries"
        }
      ],
      "title": "Service Discovery",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 21
      },
      "id": 99,
      "panels": [
        {
          "datasource": "$ds_prometheus",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "normal"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 7,
            "w": 10,
            "x": 0,
            "y": 22
          },
          "id": 105,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod, interval) (rate(prometheus_target_interval_length_seconds_sum{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]) / rate(prometheus_target_interval_length_seconds_count{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{interval}}",
              "refId": "A"
            }
          ],
          "title": "Scrape Interval",
          "type": "timeseries"
        },
        {
          "datasource": "$ds_prometheus",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "normal"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 7,
            "w": 14,
            "x": 10,
            "y": 22
          },
          "id": 104,
          "links": [],
          "options": {
            "legend": {
              "calcs": [
                "last",
                "max"
              ],
              "displayMode": "table",
              "placement": "right"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod) (rate(prometheus_target_scrapes_exceeded_sample_limit_total{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Exceeded Sample Limit",
              "refId": "A"
            },
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod) (rate(prometheus_target_scrapes_sample_duplicate_timestamp_total{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Duplicate Timestamp",
              "refId": "C"
            },
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod) (rate(prometheus_target_scrapes_sample_out_of_bounds_total{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Out Of Bounds ",
              "refId": "D"
            },
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod) (rate(prometheus_target_scrapes_sample_out_of_order_total{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Out of Order",
              "refId": "E"
            }
          ],
          "title": "Scrape Problems/s",
          "type": "timeseries"
        },
        {
          "datasource": "$ds_prometheus",
          "fieldConfig": {
            "defaults": {
              "color": {
                "fixedColor": "orange",
                "mode": "fixed"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "normal"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "blue"
                  },
                  {
                    "color": "red",
                    "value": 80
                  },
                  {
                    "color": "#EAB839",
                    "value": 90
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 6,
            "x": 0,
            "y": 29
          },
          "id": 108,
          "links": [],
          "options": {
            "legend": {
              "calcs": [
                "last",
                "max"
              ],
              "displayMode": "hidden",
              "placement": "bottom",
              "sortBy": "Last",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "editorMode": "code",
              "expr": "sum(max by (pod, scrape_job) (prometheus_target_metadata_cache_bytes{job=\"prometheus\",pod=\"$pod\"}))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "Total",
              "range": true,
              "refId": "A"
            }
          ],
          "title": "Metadata Cache Size Total",
          "type": "timeseries"
        },
        {
          "datasource": "$ds_prometheus",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "normal"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": []
          },
          "gridPos": {
            "h": 8,
            "w": 18,
            "x": 6,
            "y": 29
          },
          "id": 95,
          "links": [],
          "options": {
            "legend": {
              "calcs": [
                "last",
                "max"
              ],
              "displayMode": "table",
              "placement": "right",
              "sortBy": "Last",
              "sortDesc": true
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod, scrape_job) (prometheus_target_metadata_cache_bytes{job=\"prometheus\",pod=\"$pod\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 1,
              "legendFormat": "{{scrape_job}}",
              "refId": "A"
            }
          ],
          "title": "Metadata Cache Size",
          "type": "timeseries"
        }
      ],
      "title": "Scraping",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 22
      },
      "id": 52,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 35,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "dateTimeAsIso"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#447ebc",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Min"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "purple",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Now"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "text",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max"
                },
                "properties": [
                  {
                    "id": "custom.fillBelowTo",
                    "value": "Min"
                  },
                  {
                    "id": "custom.lineWidth",
                    "value": 0
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 0,
            "y": 23
          },
          "id": 28,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (prometheus_tsdb_head_min_time{job=\"prometheus\",pod=\"$pod\"})",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "Min",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "time() * 1000",
              "format": "time_series",
              "hide": false,
              "intervalFactor": 2,
              "legendFormat": "Now",
              "refId": "C"
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (prometheus_tsdb_head_max_time{job=\"prometheus\",pod=\"$pod\"})",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "Max",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "B",
              "step": 10
            }
          ],
          "title": "Head Time Range",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "GC Time/s"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "super-light-yellow",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 8,
            "y": 23
          },
          "id": 29,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "hidden",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (rate(prometheus_tsdb_head_gc_duration_seconds_sum{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "GC Time/s",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Head GC Time/s",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "short"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Blocks Loaded"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "orange",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 16,
            "y": 23
          },
          "id": 14,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "hidden",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (prometheus_tsdb_blocks_loaded{job=\"prometheus\",pod=\"$pod\"})",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "Blocks Loaded",
              "metric": "prometheus_local_storage_indexing_batch_sizes_sum",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Blocks Loaded",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Reloads"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Failed Compactions"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#bf1b00",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Failed Reloads"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#bf1b00",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 0,
            "y": 30
          },
          "id": 30,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (rate(prometheus_tsdb_reloads_total{job=\"prometheus\",pod=\"$pod\"}[10m]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "Reloads",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "TSDB Reloads/s",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Failed Compactions"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#bf1b00",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 8,
            "y": 30
          },
          "id": 31,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (rate(prometheus_tsdb_wal_fsync_duration_seconds_sum{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]) / rate(prometheus_tsdb_wal_fsync_duration_seconds_count{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "Fsync Latency",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (rate(prometheus_tsdb_wal_truncate_duration_seconds_sum{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]) / rate(prometheus_tsdb_wal_trunacte_duration_seconds_count{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "Truncate Latency",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "B",
              "step": 10
            }
          ],
          "title": "WAL Fsync&Truncate Latency",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Failed Compactions"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#bf1b00",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "{instance=\"demo.robustperception.io:9090\",job=\"prometheus\"}"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#bf1b00",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 16,
            "y": 30
          },
          "id": 32,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (rate(prometheus_tsdb_wal_corruptions_total{job=\"prometheus\",pod=\"$pod\"}[10m]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "WAL Corruptions",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (rate(prometheus_tsdb_reloads_failures_total{job=\"prometheus\",pod=\"$pod\"}[10m]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "Reload Failures",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "B",
              "step": 10
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "rate(prometheus_tsdb_head_series_not_found{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "Head Series Not Found",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "C",
              "step": 10
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "rate(prometheus_tsdb_compactions_failed_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "Compaction Failures",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "D",
              "step": 10
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "rate(prometheus_tsdb_retention_cutoffs_failures_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "Retention Cutoff Failures",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "E",
              "step": 10
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "rate(prometheus_tsdb_checkpoint_creations_failed_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "WAL Checkpoint Creation Failures",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "F",
              "step": 10
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "rate(prometheus_tsdb_checkpoint_deletions_failed_total{job=\"prometheus\",instance=\"$Prometheus:9090\"}[10m])",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "WAL Checkpoint Deletion Failures",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "G",
              "step": 10
            }
          ],
          "title": "TSDB Problems/s",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Failed Compactions"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#bf1b00",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Compactions"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "super-light-blue",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 0,
            "y": 37
          },
          "id": 19,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (rate(prometheus_tsdb_compactions_total{job=\"prometheus\",pod=\"$pod\"}[10m]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "Compactions",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Compactions/s",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 8,
            "y": 37
          },
          "id": 33,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "hidden",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (rate(prometheus_tsdb_compaction_duration_seconds_sum{job=\"prometheus\",pod=\"$pod\"}[10m]))",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Compaction Time/s",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Allocated bytes"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#F9BA8F",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "RSS"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#890F02",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 16,
            "y": 37
          },
          "id": 8,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (rate(prometheus_tsdb_time_retentions_total{job=\"prometheus\",pod=\"$pod\"}[10m]))",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "Time Cutoffs",
              "metric": "last",
              "refId": "A",
              "step": 10
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (rate(prometheus_tsdb_size_retentions_total{job=\"prometheus\",pod=\"$pod\"}[10m]))",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "Size Cutoffs",
              "metric": "last",
              "refId": "B",
              "step": 10
            }
          ],
          "title": "Retention Cutoffs/s",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "dtdurationms"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunk Time Range"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "yellow",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 0,
            "y": 44
          },
          "id": 27,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "hidden",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (rate(prometheus_tsdb_compaction_chunk_range_seconds_sum{job=\"prometheus\",pod=\"$pod\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_range_seconds_count{job=\"prometheus\",pod=\"$pod\"}[10m]))",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "Chunk Time Range",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "First Compaction, Avg Chunk Time Range",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "bytes"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Bytes/Sample"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "dark-blue",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 8,
            "y": 44
          },
          "id": 35,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "hidden",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (rate(prometheus_tsdb_compaction_chunk_size_bytes_sum{job=\"prometheus\",pod=\"$pod\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",pod=\"$pod\"}[10m]))",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "Bytes/Sample",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "First Compaction, Avg Bytes/Sample",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunk Samples"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 16,
            "y": 44
          },
          "id": 34,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "hidden",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (dpo) (rate(prometheus_tsdb_compaction_chunk_samples_sum{job=\"prometheus\",pod=\"$pod\"}[10m]) / rate(prometheus_tsdb_compaction_chunk_samples_count{job=\"prometheus\",pod=\"$pod\"}[10m]))",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "Chunk Samples",
              "metric": "prometheus_local_storage_series_chunks_persisted_count",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "First Compaction, Avg Chunk Samples",
          "type": "timeseries"
        }
      ],
      "title": "Compaction",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 23
      },
      "id": 63,
      "panels": [
        {
          "datasource": "$ds_prometheus",
          "description": "Time spent in each mode, per second",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "normal"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 0,
            "y": 24
          },
          "id": 24,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod, slice) (rate(prometheus_engine_query_duration_seconds_sum{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{slice}}",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Query engine timings/s",
          "type": "timeseries"
        },
        {
          "datasource": "$ds_prometheus",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "short"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 8,
            "y": 24
          },
          "id": 22,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod) (rate(prometheus_rule_group_iterations_missed_total{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "Rule group missed",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "B",
              "step": 10
            },
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod) (rate(prometheus_rule_evaluation_failures_total{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "Rule evals failed",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "C",
              "step": 10
            }
          ],
          "title": "Rule group evaulation problems/s",
          "type": "timeseries"
        },
        {
          "datasource": "$ds_prometheus",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 16,
            "y": 24
          },
          "id": 23,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod) (rate(prometheus_rule_group_duration_seconds_sum{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "Rule evaluation duration",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Evaluation time of rule groups/s",
          "type": "timeseries"
        }
      ],
      "title": "Query Engine",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 24
      },
      "id": 77,
      "panels": [
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "short"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 0,
            "y": 25
          },
          "id": 86,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod, alertmanager) (rate(prometheus_notifications_sent_total{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{alertmanager}}",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Notification Sent/s",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 8,
            "y": 25
          },
          "id": 87,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod, alertmanager) (rate(prometheus_notifications_errors_total{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]) / rate(prometheus_notifications_sent_total{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{alertmanager}}",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Notification Error Ratio",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 16,
            "y": 25
          },
          "id": 81,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod, alertmanager) (rate(prometheus_notifications_latency_seconds_sum{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]) / rate(prometheus_notifications_latency_seconds_count{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "{{alertmanager}}",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Notification Latency",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 0,
            "y": 32
          },
          "id": 85,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (prometheus_notifications_alertmanagers_discovered{job=\"prometheus\",pod=\"$pod\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "Alertmanagers",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Alertmanagers Discovered",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 8,
            "y": 32
          },
          "id": 89,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (rate(prometheus_notifications_dropped_total{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "Dropped",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Notifications Dropped/s",
          "type": "timeseries"
        },
        {
          "datasource": {
            "type": "prometheus",
            "uid": "$ds_prometheus"
          },
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green",
                    "value": null
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "none"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 7,
            "w": 8,
            "x": 16,
            "y": 32
          },
          "id": 88,
          "links": [],
          "options": {
            "legend": {
              "calcs": [],
              "displayMode": "list",
              "placement": "bottom"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (prometheus_notifications_queue_length{job=\"prometheus\",pod=\"$pod\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "Pending",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
            },
            {
              "datasource": {
                "type": "prometheus",
                "uid": "$ds_prometheus"
              },
              "expr": "max by (pod) (prometheus_notifications_queue_capacity{job=\"prometheus\",pod=\"$pod\"})",
              "format": "time_series",
              "interval": "",
              "intervalFactor": 2,
              "legendFormat": "Max",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "B",
              "step": 10
            }
          ],
          "title": "Notification Queue",
          "type": "timeseries"
        }
      ],
      "title": "Notification",
      "type": "row"
    },
    {
      "collapsed": true,
      "datasource": {
        "type": "prometheus",
        "uid": "$ds_prometheus"
      },
      "gridPos": {
        "h": 1,
        "w": 24,
        "x": 0,
        "y": 25
      },
      "id": 58,
      "panels": [
        {
          "datasource": "$ds_prometheus",
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 26
          },
          "id": 38,
          "links": [],
          "options": {
            "legend": {
              "calcs": [
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "right"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod, handler) (rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{handler}}",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "HTTP requests/s",
          "type": "timeseries"
        },
        {
          "datasource": "$ds_prometheus",
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 0,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "none"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 34
          },
          "id": 37,
          "links": [],
          "options": {
            "legend": {
              "calcs": [
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "right"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod, handler) (rate(prometheus_http_request_duration_seconds_sum{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]) / rate(prometheus_http_request_duration_seconds_count{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{handler}}",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "HTTP request latency",
          "type": "timeseries"
        },
        {
          "datasource": "$ds_prometheus",
          "description": "",
          "fieldConfig": {
            "defaults": {
              "color": {
                "mode": "palette-classic"
              },
              "custom": {
                "axisLabel": "",
                "axisPlacement": "auto",
                "barAlignment": 0,
                "drawStyle": "line",
                "fillOpacity": 10,
                "gradientMode": "none",
                "hideFrom": {
                  "legend": false,
                  "tooltip": false,
                  "viz": false
                },
                "lineInterpolation": "linear",
                "lineWidth": 1,
                "pointSize": 5,
                "scaleDistribution": {
                  "type": "linear"
                },
                "showPoints": "never",
                "spanNulls": false,
                "stacking": {
                  "group": "A",
                  "mode": "normal"
                },
                "thresholdsStyle": {
                  "mode": "off"
                }
              },
              "links": [],
              "mappings": [],
              "min": 0,
              "thresholds": {
                "mode": "absolute",
                "steps": [
                  {
                    "color": "green"
                  },
                  {
                    "color": "red",
                    "value": 80
                  }
                ]
              },
              "unit": "s"
            },
            "overrides": [
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#1F78C1",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Chunks to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#508642",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max chunks"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#052B51",
                      "mode": "fixed"
                    }
                  }
                ]
              },
              {
                "matcher": {
                  "id": "byName",
                  "options": "Max to persist"
                },
                "properties": [
                  {
                    "id": "color",
                    "value": {
                      "fixedColor": "#3F6833",
                      "mode": "fixed"
                    }
                  }
                ]
              }
            ]
          },
          "gridPos": {
            "h": 8,
            "w": 24,
            "x": 0,
            "y": 42
          },
          "id": 36,
          "links": [],
          "options": {
            "legend": {
              "calcs": [
                "lastNotNull"
              ],
              "displayMode": "table",
              "placement": "right"
            },
            "tooltip": {
              "mode": "multi",
              "sort": "none"
            }
          },
          "pluginVersion": "8.5.13",
          "targets": [
            {
              "datasource": "$ds_prometheus",
              "expr": "max by (pod, handler) (rate(prometheus_http_request_duration_seconds_sum{job=\"prometheus\",pod=\"$pod\"}[$__rate_interval]))",
              "format": "time_series",
              "intervalFactor": 2,
              "legendFormat": "{{handler}}",
              "metric": "prometheus_local_storage_memory_chunkdescs",
              "refId": "A",
              "step": 10
            }
          ],
          "title": "Time spent in HTTP requests/s",
          "type": "timeseries"
        }
      ],
      "title": "HTTP Server",
      "type": "row"
    }
  ],
  "refresh": "30s",
  "schemaVersion": 36,
  "style": "dark",
  "tags": [],
  "templating": {
    "list": [
      {
        "current": {
          "selected": false,
          "text": "default",
          "value": "default"
        },
        "hide": 0,
        "includeAll": false,
        "label": "Prometheus",
        "multi": false,
        "name": "ds_prometheus",
        "options": [],
        "query": "prometheus",
        "refresh": 1,
        "regex": "",
        "skipUrlSync": false,
        "type": "datasource"
      },
      {
        "current": {
          "selected": false,
          "text": "prometheus-main-0",
          "value": "prometheus-main-0"
        },
        "datasource": {
          "type": "prometheus",
          "uid": "$ds_prometheus"
        },
        "definition": "label_values(up{job=\"prometheus\"}, pod)",
        "hide": 0,
        "includeAll": false,
        "label": "Pod",
        "multi": false,
        "name": "pod",
        "options": [],
        "query": {
          "query": "label_values(up{job=\"prometheus\"}, pod)",
          "refId": "StandardVariableQuery"
        },
        "refresh": 2,
        "regex": "",
        "skipUrlSync": false,
        "sort": 0,
        "type": "query",
        "useTags": false
      }
    ]
  },
  "time": {
    "from": "now-3h",
    "to": "now"
  },
  "timepicker": {
    "refresh_intervals": [
      "5s",
      "10s",
      "30s",
      "1m",
      "5m",
      "15m",
      "30m",
      "1h",
      "2h",
      "1d"
    ],
    "time_options": [
      "5m",
      "15m",
      "1h",
      "6h",
      "12h",
      "24h",
      "2d",
      "7d",
      "30d"
    ]
  },
  "timezone": "",
  "title": "Prometheus Benchmark",
  "uid": "L0HBvojWz",
  "version": 1,
  "weekStart": ""
}
